package com.spider.moudle.handler;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.spider.moudle.entity.Setting;
import com.spider.moudle.entity.TaskType;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.SettingsUtil;


/**
 * 检查爬虫数据的合法性，并且修复一些爬虫数据中可修复的错误
 * @author Linbo Hu
 * @date 2017/7/19
 */
public class DataHandler {


    public static void main(String[] args) throws Exception {
        DataHandler dataHandler = new DataHandler();
        String rootString = "{\"data\":{\"publish_user_id\":\"2649816\",\"title\":\"\\u6D77\\u5357\\u4E09\\u4E9A\\uFF0C\\u4E8C\\u624B\\u8F66\\u8FC1\\u5165\\u6709\\u4EC0\\u4E48\\u7279\\u6B8A\\u7684\\u89C4\\u5B9A\\u5417_\\u6D77\\u5357\\u6C7D\\u8F66\\u8BBA\\u575B_XCAR \\u7231\\u5361\\u6C7D\\u8F66\\u4FF1\\u4E50\\u90E8\",\"content\":\"\\u6D77\\u5357\\u4E09\\u4E9A\\uFF0C\\u4E8C\\u624B\\u8F66\\u8FC1\\u5165\\u6709\\u4EC0\\u4E48\\u7279\\u6B8A\\u7684\\u89C4\\u5B9A\\u5417 9\\u7231\\u5361\\u5E01 \\u95EE\\u9898\\u63CF\\u8FF0 \\u6D77\\u5357\\u77012017\\u5E744\\u6708\\u65B0\\u51FA\\u4E86\\u4E8C\\u624B\\u8F66\\u4EA4\\u6613\\u89C4\\u5B9A\\uFF0C\\u56FD\\u56DB\\u4EE5\\u4E0A\\u6807\\u51C6\\u53EF\\u4EE5\\u8FC1\\u5165\\uFF0C\\u4E0D\\u77E5\\u6D77\\u5357\\u7684\\u5144\\u5F1F\\u4EEC\\u4E86\\u89E3\\u60C5\\u51B5\\u5417\\uFF1F\\u662F\\u5426\\u529E\\u7406\\u987A\\u5229\\uFF0C\\u6709\\u4EC0\\u4E48\\u7279\\u6B8A\\u8981\\u6C42\\u5417\\uFF1F\\u8C22\\u8C22 \\u6700\\u4F73\\u7B54\\u6848 0898\\u5F00\\u5FC398 \\u53D1\\u8868\\u4E8E2017-12-05 10:33 \\u6D77\\u5357\\u7701\\u6D77\\u53E3\\u5E02\\u65B0\\u51FA\\u4E86\\u4E8C\\u624B\\u8F66\\u4EA4\\u6613\\u89C4\\u5B9A\\uFF0C\\u4ECE2017\\u5E7412\\u67081\\u65E5\\u8D77\\u5916\\u5730\\u8F6C\\u5165\\u6D77\\u53E3\\u7684\\u4E8C\\u624B\\u8F66\\u8981\\u6C42\\u5FC5\\u987B\\u56FD\\u4E94\\u6216\\u4EE5\\u4E0A\\u6807\\u51C6\\u624D\\u53EF\\u4EE5\\u8FC1\\u5165[\\u67E5\\u770B\\u8BE6\\u7EC6]\",\"publish_user\":\"coke2099\",\"spider_time\":\"2018-01-08 13:00:37\",\"publish_time\":\"2017-12-04 10:32:00\",\"like_nodes\":[],\"id\":\"30806792\",\"is_pic\":null,\"read_count\":16335,\"area\":{\"country\":\"\\u4E2D\\u56FD\",\"province\":\"\\u6D77\\u5357\\u7701\",\"city\":null,\"lon\":\"110.348630\",\"lat\":\"20.019970\"},\"like_count\":0,\"txpath\":\"\",\"reply_count\":4,\"reply_nodes\":[{\"publish_user_id\":\"7423518\",\"title\":\"\",\"content\":\"\\u5DF2\\u88AB\\u9009\\u4E3A\\u6700\\u4F73\\u7B54\\u6848 \\u5F15\\u7528 coke2099 \\u4E8E 17-12-04 10:32 \\u53D1\\u8868\\u7684\\u56DE\\u590D \\u6D77\\u5357\\u77012017\\u5E744\\u6708\\u65B0\\u51FA\\u4E86\\u4E8C\\u624B\\u8F66\\u4EA4\\u6613\\u89C4\\u5B9A\\uFF0C\\u56FD\\u56DB\\u4EE5\\u4E0A\\u6807\\u51C6\\u53EF\\u4EE5\\u8FC1\\u5165\\uFF0C\\u4E0D\\u77E5\\u6D77\\u5357\\u7684\\u5144\\u5F1F\\u4EEC\\u4E86\\u89E3\\u60C5\\u51B5\\u5417\\uFF1F\\u662F\\u5426\\u529E\\u7406\\u987A\\u5229\\uFF0C\\u6709\\u4EC0\\u4E48\\u7279\\u6B8A\\u8981\\u6C42\\u5417\\uFF1F\\u8C22\\u8C22 \\u6D77\\u5357\\u7701\\u6D77\\u53E3\\u5E02\\u65B0\\u51FA\\u4E86\\u4E8C\\u624B\\u8F66\\u4EA4\\u6613\\u89C4\\u5B9A\\uFF0C\\u4ECE2017\\u5E7412\\u67081\\u65E5\\u8D77\\u5916\\u5730\\u8F6C\\u5165\\u6D77\\u53E3\\u7684\\u4E8C\\u624B\\u8F66\\u8981\\u6C42\\u5FC5\\u987B\\u56FD\\u4E94\\u6216\\u4EE5\\u4E0A\\u6807\\u51C6\\u624D\\u53EF\\u4EE5\\u8FC1\\u5165\",\"publish_user\":\"0898\\u5F00\\u5FC398\",\"spider_time\":\"2018-01-08 13:00:37\",\"publish_time\":\"2017-12-05 10:33:00\",\"like_nodes\":[],\"id\":\"\",\"is_pic\":null,\"read_count\":0,\"area\":null,\"like_count\":0,\"txpath\":\"\",\"reply_count\":0,\"reply_nodes\":[],\"url\":\"http:\\/\\/www.xcar.com.cn\\/\\/bbs\\/viewthread.php?tid=30806792#table_542789365\",\"reproduce_count\":0,\"publish_user_photo\":\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/74\\/23\\/518_120.jpg?r=20180108\",\"ancestor_id\":\"30806792\",\"reproduce_nodes\":[],\"publish_user_jsonid\":\"\\u7231\\u5361\\u8BBA\\u575B_7423518\",\"parent_id\":\"30806792\",\"img_urls\":[\"http:\\/\\/www.xcar.com.cn\\/bbs\\/images\\/common\\/back.gif\"],\"video_urls\":[],\"dislike_count\":0},{\"publish_user_id\":\"1673335\",\"title\":\"\",\"content\":\"\\u53EA\\u8981\\u662F\\u6309\\u89C4\\u5B9A\\u7684\\u624B\\u7EED\\u5C31\\u53EF\\u4EE5\\u4E86.\",\"publish_user\":\"thills\",\"spider_time\":\"2018-01-08 13:00:37\",\"publish_time\":\"2017-12-06 09:21:00\",\"like_nodes\":[],\"id\":\"\",\"is_pic\":null,\"read_count\":0,\"area\":null,\"like_count\":0,\"txpath\":\"\",\"reply_count\":0,\"reply_nodes\":[],\"url\":\"http:\\/\\/www.xcar.com.cn\\/\\/bbs\\/viewthread.php?tid=30806792#table_542846410\",\"reproduce_count\":0,\"publish_user_photo\":\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/16\\/73\\/335_120.jpg?r=20180108\",\"ancestor_id\":\"30806792\",\"reproduce_nodes\":[],\"publish_user_jsonid\":\"\\u7231\\u5361\\u8BBA\\u575B_1673335\",\"parent_id\":\"30806792\",\"img_urls\":[],\"video_urls\":[],\"dislike_count\":0},{\"publish_user_id\":\"1036738\",\"title\":\"\",\"content\":\"\\u65B0\\u89C4\\u5DF2\\u51FA\\uFF0C\\u5FC5\\u987B\\u56FD\\u4E94\",\"publish_user\":\"DVD1\",\"spider_time\":\"2018-01-08 13:00:37\",\"publish_time\":\"2017-12-06 09:39:00\",\"like_nodes\":[],\"id\":\"\",\"is_pic\":null,\"read_count\":0,\"area\":null,\"like_count\":0,\"txpath\":\"\",\"reply_count\":0,\"reply_nodes\":[],\"url\":\"http:\\/\\/www.xcar.com.cn\\/\\/bbs\\/viewthread.php?tid=30806792#table_542847787\",\"reproduce_count\":0,\"publish_user_photo\":\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/10\\/36\\/738_120.jpg?r=20180108\",\"ancestor_id\":\"30806792\",\"reproduce_nodes\":[],\"publish_user_jsonid\":\"\\u7231\\u5361\\u8BBA\\u575B_1036738\",\"parent_id\":\"30806792\",\"img_urls\":[],\"video_urls\":[],\"dislike_count\":0},{\"publish_user_id\":\"2649816\",\"title\":\"\",\"content\":\"\\u5F15\\u7528 0898\\u5F00\\u5FC398 \\u4E8E 17-12-05 10:33 \\u53D1\\u8868\\u7684\\u56DE\\u590D \\u6D77\\u5357\\u7701\\u6D77\\u53E3\\u5E02\\u65B0\\u51FA\\u4E86\\u4E8C\\u624B\\u8F66\\u4EA4\\u6613\\u89C4\\u5B9A\\uFF0C\\u4ECE2017\\u5E7412\\u67081\\u65E5\\u8D77\\u5916\\u5730\\u8F6C\\u5165\\u6D77\\u53E3\\u7684\\u4E8C\\u624B\\u8F66\\u8981\\u6C42\\u5FC5\\u987B\\u56FD\\u4E94\\u6216\\u4EE5\\u4E0A\\u6807\\u51C6\\u624D\\u53EF\\u4EE5\\u8FC1\\u5165 \\u975E\\u5E38\\u611F\\u8C22\\uFF01\",\"publish_user\":\"coke2099\",\"spider_time\":\"2018-01-08 13:00:37\",\"publish_time\":\"2017-12-06 09:42:00\",\"like_nodes\":[],\"id\":\"\",\"is_pic\":null,\"read_count\":0,\"area\":null,\"like_count\":0,\"txpath\":\"\",\"reply_count\":0,\"reply_nodes\":[],\"url\":\"http:\\/\\/www.xcar.com.cn\\/\\/bbs\\/viewthread.php?tid=30806792#table_542847977\",\"reproduce_count\":0,\"publish_user_photo\":\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/26\\/49\\/816_120.jpg?r=20180108\",\"ancestor_id\":\"30806792\",\"reproduce_nodes\":[],\"publish_user_jsonid\":\"\\u7231\\u5361\\u8BBA\\u575B_2649816\",\"parent_id\":\"30806792\",\"img_urls\":[\"http:\\/\\/www.xcar.com.cn\\/bbs\\/images\\/common\\/back.gif\"],\"video_urls\":[],\"dislike_count\":0}],\"url\":\"http:\\/\\/www.xcar.com.cn\\/\\/bbs\\/viewthread.php?tid=30806792\",\"reproduce_count\":0,\"publish_user_photo\":\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/26\\/49\\/816_120.jpg?r=20180108\",\"ancestor_id\":\"\",\"reproduce_nodes\":[],\"publish_user_jsonid\":\"\\u7231\\u5361\\u8BBA\\u575B_2649816\",\"parent_id\":\"\",\"img_urls\":[\"http:\\/\\/image1.xcarimg.com\\/attachments\\/my\\/0\\/74\\/23\\/518_50.jpg\"],\"video_urls\":[],\"dislike_count\":0}}";

        JSONObject object = dataHandler.handle(JSON.parseObject(rootString), SettingsUtil.getSettingBySite("AiKa"), TaskType.SPEECH);
        System.out.println(JSON.toJSONString(object, SerializerFeature.DisableCircularReferenceDetect));
    }

    /**
     * 处理抓下来的数据是否符合要求
     * @param root
     * @param setting
     * @param taskType
     * @return
     */
    public JSONObject handle(JSONObject root, Setting setting, TaskType taskType) throws Exception {

        switch (taskType) {
            case USER:{
                //网站的中文名(拼凑 publish_user_jsonid 有用)
                String name = setting.getName();
                JSONObject data = root.getJSONObject("data");
                String json_id = data.getString("json_id");
                String publish_user_id = data.getString("publish_user_id");
                if(CommonUtil.isNull(json_id) && CommonUtil.isNotNull(publish_user_id)) {
                    data.put("json_id", name + "_" + publish_user_id);
                }

                //关注
                Integer follow_count = data.getInteger("follow_count");
                JSONArray follow_details = data.getJSONArray("follow_details");
                Integer follow_details_size = follow_details.size();
                if(follow_count == 0 && follow_details_size > 0) {
                    data.put("follow_count", follow_details_size);
                }
                //粉丝
                Integer fans_count = data.getInteger("fans_count");
                JSONArray fans_details = data.getJSONArray("fans_details");
                Integer fans_details_size = fans_details.size();
                if(fans_count == 0 && fans_details_size > 0) {
                    data.put("fans_count", fans_details_size);
                }
                //朋友
                Integer friend_count = data.getInteger("friend_count");
                JSONArray friend_details = data.getJSONArray("friend_details");
                Integer friend_details_size = friend_details.size();
                if(friend_count == 0 && friend_details_size > 0) {
                    data.put("friend_count", friend_details_size);
                }
                return root;
            }
            case SPEECH:{
                JSONObject data = root.getJSONObject("data");
                String time = data.getString("publish_time");


                //判断publish_time是否存在， publish_time不存在的数据对于speech是不符合规范的
                if (CommonUtil.isNull(time)) {
                    throw new Exception(setting.getName() + " publish_time 不能为空");
                }


                String id = data.getString("id");
                if(CommonUtil.isNull(id)) {
                    throw new Exception(setting.getName() + " id 不能为空");
                }

                //如果主贴中的 ancestor_id 为空，则将主贴中的 ancestor_id 补全
                String ancestor_id = data.getString("ancestor_id");
                if(CommonUtil.isNotNull(ancestor_id)) {
                    data.put("ancestor_id", "");
                }


                //如果主贴中的 parent_id 为空，则将主贴中的 parent_id 补全
                String parent_id = data.getString("parent_id");
                if(CommonUtil.isNotNull(parent_id)) {
                    data.put("parent_id", id);
                }

                //网站的中文名(拼凑 publish_user_jsonid 有用)
                String name = setting.getName();
                //当数据里面没有publish_user_jsonid时，手动拼凑publish_user_jsonid
                String publish_user_jsonid = data.getString("publish_user_jsonid");
                String publish_user_id = data.getString("publish_user_id");
                if(CommonUtil.isNull(publish_user_jsonid) && CommonUtil.isNotNull(publish_user_id)) {
                    data.put("publish_user_jsonid", name + "_" + publish_user_id);
                }

                //去掉title中的html标签
                String title = data.getString("title");
                if(CommonUtil.isNotNull(title)) {
                    data.put("title", CommonUtil.filtHtml(title).trim());
                }


                //去掉content中的html标签
                String content = data.getString("content");
                if(CommonUtil.isNotNull(content)) {
                    data.put("content", CommonUtil.filtHtml(content).trim());
                }

                //回复节点
                JSONObject area = data.getJSONObject("area");
                JSONArray reply_nodes = data.getJSONArray("reply_nodes");
                for(int i = 0; i < reply_nodes.size(); i++) {
                    //这是一个回复实体
                    JSONObject reply_node = reply_nodes.getJSONObject(i);

                    String reply_id = reply_node.getString("id");


                    //如果回复中的 ancestor_id 为空，则将回复中的 ancestor_id 补全
                    String reply_ancestor_id = reply_node.getString("ancestor_id");
                    if(CommonUtil.isNull(reply_ancestor_id)) {
                        reply_node.put("ancestor_id", id);
                    }


                    //如果回复中的 parent_id 为空，则将回复中的 parent_id 补全
                    String reply_parent_id = reply_node.getString("parent_id");
                    if(CommonUtil.isNull(reply_parent_id)) {
                        reply_node.put("parent_id", id);
                    }

                    //回复中 ancestor_id 应该等于 parent_id
                    if(!reply_node.getString("ancestor_id").equals(reply_node.getString("parent_id"))) {
                        throw new Exception(setting.getName() + " 对于回复来说 parent_id 应该等于 ancestor_id");
                    }

                    //如果回复中的publish_user_jsonid为空，则将回复中的publish_user_jsonid补全
                    String reply_publish_user_jsonid = reply_node.getString("publish_user_jsonid");
                    String reply_publish_user_id = reply_node.getString("publish_user_id");
                    if(CommonUtil.isNull(reply_publish_user_jsonid) && CommonUtil.isNotNull(reply_publish_user_id)) {
                        reply_node.put("publish_user_jsonid", name + "_" + reply_publish_user_id);
                    }


                    //去掉title中的html标签
                    String reply_title = reply_node.getString("title");
                    if(CommonUtil.isNotNull(reply_title)) {
                        reply_node.put("title", CommonUtil.filtHtml(reply_title).trim());
                    }

                    //去掉content中的html标签
                    String reply_content = reply_node.getString("content");
                    if(CommonUtil.isNotNull(reply_content)) {
                        reply_node.put("content", CommonUtil.filtHtml(reply_content).trim());
                    }



                    //如果回复中的 area 为空，则将回复中的 area 补全
                    JSONObject reply_area = reply_node.getJSONObject("area");
                    if(CommonUtil.isNull(reply_area) && CommonUtil.isNotNull(area)) {
                        reply_node.put("area", area);
                    }



                    JSONArray reply_reply_nodes = reply_node.getJSONArray("reply_nodes");
                    //回复的回复节点
                    for(int j = 0; j < reply_reply_nodes.size(); j++) {
                        //回复的回复实体
                        JSONObject reply_reply_node = reply_reply_nodes.getJSONObject(j);


                        //如果回复的回复中的 ancestor_id 为空，则将回复的回复中的 ancestor_id 补全
                        String repy_reply_ancestor_id = reply_reply_node.getString("ancestor_id");
                        if(CommonUtil.isNull(repy_reply_ancestor_id)) {
                            reply_reply_node.put("ancestor_id", id);
                        }

                        //如果回复的回复中的 parent_id 为空，则将回复的回复中的 parent_id 补全
                        String reply_reply_parent_id = reply_reply_node.getString("parent_id");
                        if(CommonUtil.isNull(reply_reply_parent_id)) {
                            reply_reply_node.put("parent_id", reply_id);
                        }


                        String reply_reply_publish_user_jsonid = reply_reply_node.getString("publish_user_jsonid");
                        String reply_reply_publish_user_id = reply_reply_node.getString("publish_user_id");
                        if(CommonUtil.isNull(reply_reply_publish_user_jsonid) && CommonUtil.isNotNull(reply_reply_publish_user_id)) {
                            reply_reply_node.put("publish_user_jsonid", name + "_" + reply_reply_publish_user_id);
                        }


                        //去掉title中的html标签
                        String reply_reply_title = reply_reply_node.getString("title");
                        if(CommonUtil.isNotNull(reply_reply_title)) {
                            reply_reply_node.put("title", CommonUtil.filtHtml(reply_reply_title).trim());
                        }

                        //去掉content中的html标签
                        String reply_reply_content = reply_reply_node.getString("content");
                        if(CommonUtil.isNotNull(reply_reply_content)) {
                            reply_reply_node.put("content", CommonUtil.filtHtml(reply_reply_content).trim());
                        }


                        //如果回复的回复中的 area 为空，则将回复的回复中的 area 补全
                        JSONObject reply_reply_area = reply_reply_node.getJSONObject("area");
                        if(CommonUtil.isNull(reply_reply_area) && CommonUtil.isNotNull(area)) {
                            reply_reply_node.put("area", area);
                        }

                    }
                }
                return root;
            }
            default:{
                throw new Exception(setting.getName() + " task type is unavailable");
            }
        }
    }


}
